# -*- coding: utf-8 -*-
import scrapy
import pandas as pd
from scrapy import FormRequest


class CarspecsSpider(scrapy.Spider):
    name = 'carspecs'
    start_urls = ['https://www.carfolio.com/car-makes/']
    dic = {}
    data_mk = ['ACURA', 'ALFA ROMEO', 'ASTON MARTIN', 'AUDI', 'BEIJING AUTO',
               'BENTLEY', 'BMW', 'BRILLIANCE', 'BUICK', 'BYD', 'CADILLAC',
               'CATERHAM', 'CHANA', 'CHANGHE', 'CHERY', 'CHEVROLET', 'CHRYSLER',
               'CITROËN', 'DACIA', 'DAEWOO', 'DAIHATSU', 'DATSUN', 'DODGE',
               'DONGFENG', 'DS', 'FAW', 'FERRARI', 'FIAT', 'FORD', 'GEELY',
               'GREAT WALL', 'GREEN WHEEL', 'HAIMA', 'HONDA', 'HUDSON', 'HUMMER',
               'HYUNDAI', 'INFINITI', 'JAC', 'JAGUAR', 'JEEP', 'KARRY', 'KIA',
               'LANCIA', 'LAND ROVER', 'LEXUS', 'LIFAN', 'LINCOLN', 'MAHINDRA',
               'MASERATI', 'MAZDA', 'MCLAREN', 'MERCEDES-BENZ', 'MERCURY', 'MG',
               'MINI', 'MITSUBISHI', 'NISSAN', 'PEUGEOT', 'PLYMOUTH', 'PONTIAC',
               'PORSCHE', 'RAM', 'RENAULT', 'ROLLS-ROYCE', 'ROVER', 'SATURN',
               'SCION', 'SEAT', 'SKODA', 'SMART', 'STUDEBAKER', 'SUBARU',
               'SUZUKI', 'TESLA', 'TOYOTA', 'UNSPECIFIED', 'VOLKSWAGEN', 'VOLVO',
               'WESTFIELD', 'ZEDRIV', 'ACURA-GUANGZHOU', 'AIWAYS', 'ALPINA',
               'ARCFOX', 'AUDI-FAW', 'BAIC YINXIANG', 'BAOJUN', 'BESTURN', 'BISU',
               'BJEV', 'BMW-BRILLIANCE', 'BORGWARD', 'BUFORI', 'BUICK-SHANGHAI',
               'BYVIN', 'CADILLAC-SHANGHAI', 'CHANGAN', 'CHANGJIANG',
               'CHEVROLET-SHANGHAI', 'CHEVROLET-WULING', 'CHRYSLER-FUTIAN',
               'CIIMO', 'CITROEN-DONGFENG', 'COWIN', 'DAYUN', 'DEARCC', 'DENZA',
               'DODGE-SOUEAST', 'DONGFENG-ZNA', 'DORCEN', 'DS-CHANGAN', 'ENOVATE',
               'ENRANGER (YINGZHI)', 'EVERUS', 'EXEED', 'FIAT-GUANGZHOU', 'FODAY',
               'FORD-CHANGAN', 'FORD-JMC', 'FOTON', 'FUJIAN', 'FUKANG',
               'GEELY-EMGRAND', 'GEELY-ENGLON', 'GEELY-GLEAGLE', 'GEOMETRY',
               'GMC', 'GOLDEN DRAGON', 'GONOW', 'GUANGZHOU AUTO', 'HAFEI',
               'HANTENG', 'HAWTAI', 'HIPHI', 'HONDA-DONGFENG', 'HONDA-GUANGZHOU',
               'HONGQI', 'HONGXING', 'HORKI', 'HUANGHAI', 'HUNKT', 'HYCAN',
               'HYUNDAI-BEIJING', 'INFINITI-DONGFENG', 'ISUZU-JIANGXI',
               'ISUZU-QINGLING', 'JAC-ANCHI', 'JAGUAR-CHERY', 'JEEP-GUANGZHOU',
               'JETOUR', 'JETTA', 'JIABAO (SICHUAN)', 'JIANGLING EV',
               'JIANGNAN (HUNAN)', 'JINBEI', 'JINCHENG', 'JMC', 'JONWAY',
               'JOYLONG', 'KAICHENG', 'KANDI', 'KARMA', 'KAWEI', 'KIA-YUEDA',
               'KTM', 'LADA', 'LAMBORGHINI', 'LAND ROVER-CHERY', 'LANDWIND',
               'LEAPMOTOR', 'LEOPAARD', 'LEVC', 'LEVDEO', 'LI', 'LIANHUA',
               'LINCOLN-CHANGAN', 'LINGBAO', 'LINKTOUR', 'LOTUS', 'LUXGEN',
               'LYNK & CO', 'MAPLE', 'MAXUS', 'MAYBACH', 'MAZDA-CHANGAN',
               'MAZDA-FAW', 'MERCEDES-BEIJING', 'MERCEDES-FUJIAN', 'MINAN AUTO',
               'MINGJUN', 'MITSUBISHI-GAC', 'MITSUBISHI-SOUEAST', 'MITSUOKA',
               'MORGAN', 'NETA', 'NEVS', 'NEW-DADI (XINDADI)', 'NIO',
               'NISSAN-DONGFENG', 'NOBLE', 'OLEY', 'OPEL', 'ORA', 'OSSAN',
               'PAGANI', 'PEUGEOT-DONGFENG', 'POCCO', 'POLESTAR', 'PROTON',
               'QIANTU', 'QOROS', 'RELY (WEILIN)', 'RENAULT-DONGFENG', 'RIICH',
               'ROEWE', 'SAAB', 'SERES', 'SHAANXI (HANJIANG)',
               'SHANGHAI MAPLE (SMA)', 'SHOUWANG', 'SHUANGHUAN', 'SINGULATO',
               'SINOGOLD', 'SITECH', 'SKODA-SHANGHAI', 'SKYWELL', 'SOUEAST',
               'SPYKER', 'SSANGYONG', 'SUDA', 'SUZUKI-CHANGAN', 'SUZUKI-CHANGHE',
               'SWM', 'TANTUS (TIANYU)', 'TIANMA', 'TIANQI MEIYA',
               'TONGJIA (SHAANXI)', 'TOYOTA-FAW', 'TOYOTA-GUANGZHOU', 'TRAUM',
               'UAZ', 'VENUCIA', 'VICTORY (YICHY)', 'VIEW', 'VOLKSWAGEN-FAW',
               'VOLKSWAGEN-SHANGHAI', 'VOLVO-CHANGAN', 'VOLVO-GEELY', 'WEY',
               'WM MOTOR', 'WULING', 'XIALI', 'XINKAI AUTO', 'XPENG', 'YEMA',
               'YOUNGMAN', 'YUDO', 'YULON', 'ZHIDOU', 'ZHONGHUA', 'ZINORO',
               'ZOTYE', 'ZX AUTO (ZHONGXING)']

    # yield FormRequest(url=self.start_urls, callback=self.parse_maker)

    def parse(self, response):
        for qq in response.xpath('//div/a[@class="man"]'):
            if qq.xpath('text()').get().upper() in self.data_mk:
                url = "https://www.carfolio.com" + qq.xpath('@href').get()
                yield scrapy.Request(url, callback=self.parse_mod)
            else:
                pass

    def parse_mod(self, response):
        bb = response.xpath('//div[@class="grid"]/div/@id').getall()  # Get models Ids
        for x in bb:
            yield {x :{
                'manufacturer': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="card-head"]/a/span/span[@class="manufacturer"]/text()').get(),
                'market': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="card-head"]/abbr[@class="market"]/@title').get(),
                'year': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="card-head"]/a/span/span[@class="Year" or @class="model-year"]/text()').get(),
                'model': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="card-head"]/a/span/span[@class="model name"]/text()').get(),
                'doors': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="detail"]/div[@class="rest"]/span[@class="doors"]/text()').get(),
                'body': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="detail"]/div[@class="rest"]/span[@class="body"]/text()').get(),
                'layout': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="detail"]/div[@class="rest"]/span[@class="layout"]/abbr/text()').get(),
                'gear_box': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="detail"]/div[@class="rest"]/span[@class="gearbox"]/abbr/text()').get(),
                'volume': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="detail"]/div[@class="rest"]/span[@class="volume"]/text()').get(),
                'power': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="detail"]/div[@class="rest"]/span[@class="power"]/text()').get(),
                'emissions': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="detail"]/div[@class="rest"]/span[@class="emissions"]/text()').get(),
                'weight': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="detail"]/div[@class="rest"]/span[@class="weight"]/text()').get(),
                'fuel': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="footer"]/div[@class="footer"]/div[contains(@class, "fuel")]/@title').get(),
                'turbo': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="footer"]/div[@class="aspiration turbo"]/@title').get(),
                'others': response.xpath(
                    '//div[@class="grid"]/div[@id="' + x + '"]/div[@class="footer"]/small/text()').get(),
            }
            }
